#!/usr/bin/env python
#coding=utf-8
import httplib
import re
import DBConn
import urllib
import Casing
from BeautifulSoup import BeautifulSoup
class Spider(object):
    def __init__(self,callback):
    #def __init__(self):
        self.strMsgBlock = ""
        self.callback = callback
    def Create(self,str_1):
        self.strID = str(str_1)
        conn = httplib.HTTPConnection("www.cndev.org")
        conn.request("GET", "/forum/msg.aspx?pid="+self.strID)
        sock = conn.getresponse()
        self.htm = sock.read()
        self.soup = BeautifulSoup(self.htm)
        if self.IFErrorPage() == False:
            self.GetMsgsBlock()
            self.GetMsgBlock()     
    def GetID(self):
        return self.strID
    def GetMsgContent(self):
        self.soup = BeautifulSoup(self.htm)
        ocontent = self.soup.find("span",id='ctl00_placeContent_abcMsgBody')
        if ocontent.string == u"&lt;空&gt;":
            content = u"空"
        else:
            content1 = str(ocontent)
            content = unicode(content1, "utf-8") 
            content = content[41:-7]       
        return content
    def GetAllContent():
        return self.htm
    def GetMsgsBlock(self):
        dll = self.soup.find("div", { "class" : "forum-msg-block" })
        self.strMsgsBlock = str(dll)
        return self.strMsgsBlock
    def GetMsgBlock(self):
        self.soup = BeautifulSoup(self.strMsgsBlock)
        temp = self.soup.b.parent.parent
        self.strMsgBlock = str(temp)
        return self.strMsgBlock
    def GetTitle(self):
        self.soupMsg = BeautifulSoup(self.strMsgBlock)
        strTitle1 = str(self.soupMsg.b)
        strTitle = unicode(strTitle1, "utf-8")
        return strTitle[3:-4]
    def GetTime(self):
        strTitle =  self.soupMsg.nobr.string
        return strTitle
    def GetWords(self):
        temp = self.soupMsg.find('i')
        strRes = ""
        if temp == None:
            strRes = str(0)
        else:
            strWords = str(temp)
            strRes = strWords[4:-8]
        return strRes
    def GetAuthor(self):
        oAuthor = self.soupMsg.find('a',href=re.compile('^/main/user_detail.aspx.*'))
        if oAuthor == None:
            strAuthor = u"Guest"
        else:
             strAuthor = oAuthor.string
        return strAuthor
    def GetReadTimes(self):
        user_detail = self.soupMsg.find('a',href=re.compile('^/main/user_detail.aspx.*'))
        if user_detail == None:
            strRes = u'0'
        else: 
            strTemp = user_detail.nextSibling
            strReadTimes = strTemp.strip(' ')
            strRes = strReadTimes[1:-1]
        return strRes
    def GetImage(self):
        oTemp = self.soupMsg.find('img',src=re.compile('^/_res/faces/*'))
        strRes = ""
        if oTemp ==None:
            strRes = u"0"
        else:
            strTemp = str(oTemp)
            strRes = strTemp[22:34]
        return strRes
    def GetParentID(self):
        self.soup = BeautifulSoup(self.strMsgsBlock)
        self.soup.prettify()
        #oParent = soup.b.parent.parent.parent
        oSelf = self.soup.find('li',id="msg_"+self.strID)
        oParent = oSelf.parent.findPreviousSiblings("li")
        self.soup = BeautifulSoup(str(oParent))
        ooParent = self.soup.find('a',href=re.compile('^/forum/msg.aspx*'))
        strRes = ""
        if  ooParent == None:   
            strRes = 0
        else:
            strParent = ooParent["href"] 
            strResA = strParent[20:]
            strRes = int(strResA)
            #strRes = strRes.encode("utf-8")
        return strRes
    def InitDataFromCndev(self):
        #try:        
        TempID = self.GetLastMsgID()
        rtn = [0,0]
        callbackrtn=[0,0]
        strMAXMsgID = self.GetMaxMsgID()
        rtn[0] = int(TempID) - int(strMAXMsgID)
        callbackrtn[0] = rtn[0]
        callbackrtn[1] = 0
        #r = self.assignThread(int(strMAXMsgID)+1,int(self.GetLastMsgID()))
        r = [int(strMAXMsgID)+1,int(self.GetLastMsgID())]
        print r
        def f(li):
            for MsgID in range(li[0],li[1]+1): 
                try:         
                    self.Create(str(MsgID)) 
                    if self.IFErrorPage() == False:
                        strTempMsgID = self.GetID()
                        addtxt = """insert into main_data(msg_id,msg_parentid,msg_baseid,msg_boradid,msg_title,msg_content,msg_content_words,msg_time,msg_author,msg_image,msg_readtimes) values("""
                        addtxt = addtxt + strTempMsgID +','
                        addtxt = addtxt + str(self.GetParentID()) + ','
                        addtxt = addtxt + str(self.GetBaseMsgID()) + ','
                        addtxt = addtxt + "'" + self.GetBoardName() + "',"
                        addtxt = addtxt + "'" + self.GetTitle().replace("'","''") + "',"
                        addtxt = addtxt + "'" + self.GetMsgContent().replace("'","''") + "',"
                        addtxt = addtxt + self.GetWords() + ','
                        addtxt = addtxt + "'" + self.GetTime() + "',"
                        addtxt = addtxt + "'" + self.GetAuthor() + "',"
                        addtxt = addtxt + "'" + self.GetImage() + "',"
                        addtxt = addtxt + self.GetReadTimes() + ")"
                        #print addtxt.encode('utf-8')
                        DBConn.DBConn().ExecSQL(addtxt) 
                        print strTempMsgID
                        callbackrtn[1] = callbackrtn[1]+1
                        self.callback(callbackrtn)
                except:
                    print str(MsgID)+"error"
                    rtn[1] = rtn[0]+1
        #这里用的不对，应该是d = Casing.Casing(f,r),但是调试不出来。哪位大牛给指点一下？
        #for i in range(0,len(r)):
        d = Casing.Casing(f(r))
        d.start_thread()
     
        return rtn
    def assignThread(self,begin,end):
        THREADCOUNT = 10
        li = []
        num = end-begin
        if num <10:
            return [(begin,end+1)]
        else:
            div = num/THREADCOUNT
            mod = num%THREADCOUNT
            for i in range(1,THREADCOUNT+1):
                beg =begin+(i-1)*div 
                if i== THREADCOUNT:
                    li.append((beg,beg+div+mod+1))
                else:
                    li.append((beg,beg+div))
            return li

            
    def GetLastMsgID(self):
        URL_ADDRESS = "http://www.cndev.org/forum/msgs_recent.aspx"
        #URL_ADDRESS = "http://localhost/list.html"
        conn = httplib.HTTPConnection("www.cndev.org")
        #conn = httplib.HTTPConnection("localhost")
        conn.request("GET", URL_ADDRESS)
        #sock_Rece = conn.getresponse()    
        sock_Rece = urllib.urlopen(URL_ADDRESS)
        htmlRecent = sock_Rece.read()
        soupRecent = BeautifulSoup(htmlRecent)
        oRecent = soupRecent.find('div',id="page-content")
        strRecentDIV = str(oRecent)#[23:-6]
        soupRecentDIV = BeautifulSoup(strRecentDIV)
        temp = str(soupRecentDIV.tr)
        soupRecentTR = BeautifulSoup(temp)
        #strRecentA = soupRecentTR.a#find('a',href = re.compile('^/forum/msg.aspx?pid*'))
        strRes =  soupRecentTR('a')[0]['href']
        self.LastMsgID = strRes[20:] 
        return str(self.LastMsgID)
    def GetBoardName(self):
        self.soup = BeautifulSoup(self.htm)
        oBoardName = self.soup.findAll('a',href = re.compile('^/forum/bbs.aspx*'))
        strBoardName = oBoardName[0]['href']
        return strBoardName[22:]
    def GetBaseMsgID(self):
        if self.GetParentID() == 0:
            iRes = int(self.strID)
        else:
            self.soup = BeautifulSoup(self.strMsgsBlock)
            oBaseMsg = self.soup.findAll('ul',id = re.compile('_list$'))
            strBaseMsgTemp = oBaseMsg[0]['id']
            iRes = int(strBaseMsgTemp[4:-6])
        return iRes
    def IFErrorPage(self):
        self.soup = BeautifulSoup(self.htm)
        if self.soup.title.string =='CNDEV.ORG - ERROR':
            return True
        else:
            return False
    def GetMaxMsgID(self):
        sqltext = 'select max(msg_id) from main_data'
        res = DBConn.DBConn().SelectOne(sqltext)
        intRes = int(res[0])
        return intRes
"""
if __name__ == '__main__':
    spider  = Spider()
    spider.InitDataFromCndev()
"""